package zunge.lbe1;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;

import zunge.tools.Tools;

/**
 * Flow3: Generate SVD input for collaborative prediction
 * Method4: u,a,p,[u-p] -> d
 * Method5: u,a,p,[u-ap] -> d
 * Method6: u,a,p,[u-ap] -> d (Top 100 apps only)
 */
public class Flow3 {

	List<DataFold> dataFolds;
	
	public Flow3(List<DataFold> dataFolds){
		this.dataFolds = dataFolds;
	}
	
	public static final int N = 100;//#dimensions used in irlba
	
	/** label = map, matrix, uf, mf, record, prediction, log, svd, lltrain, lltest, llmodel, llout, lllog*/
	public static String getMFile(int method, int v, String tmpDir, String label){return tmpDir+"v"+v+"_m"+method+"_"+label+".txt";}
	public static String getMResultFile(int method, String resultDir){return resultDir+"m"+method+".txt";}
	
	/** m_id = getM_id(method,aid,pid) */
	public static long getM_id(int method, int aid, int pid) throws IOException{
		switch(method){
		case 4:
			return pid;
		case 5:
			return aid * Data.P + pid - 1;
		case 6:
			return aid * Data.P + pid - 1;
		default:
			throw new IOException("wrong method");	
		}
	}
	/** e_id = getE_id(uid, mid, mMap.size()) */
	public static long getE_id(int uid, int mid, int mMapSize){
		return uid * mMapSize + mid - 1;
	}
	class UM{
		public int uid;
		public int mid;
		public UM(int uid, int mid){
			this.uid = uid;
			this.mid = mid;
		}
		@Override
		public boolean equals(Object obj) {
			UM um = (UM)obj;
			return this.uid == um.uid && this.mid == um.mid;
		}
		@Override
		public int hashCode() {
			return uid + mid * 100000;
		}
	}
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	
	public void gen(ScriptGen sg) throws IOException{
		List<String> scripts = new ArrayList<String>();
		
		for(DataFold dataFold:dataFolds){
			genM(4, dataFold, scripts, sg);
			//genM(5, dataFold, scripts, sg);
			genM(6, dataFold, scripts, sg);
		}
		
		sg.splitScript(scripts, Data.CORES, "flow3", "flow3.sh",
				dataFolds.get(0).data.getScriptDir(), dataFolds.get(0).data.getTempDir());
		
	}
	
	void genM(int method, DataFold dataFold, List<String> scripts, ScriptGen sg) throws IOException{
		
		
		HashMap<Long,Integer> mMap = new HashMap<Long,Integer>();
		int uid,aid,pid,mid,did;long m_id;
		String line;
		
		//Collect Matrix Statistics
		
		BufferedReader br = new BufferedReader(new FileReader(dataFold.getTrainSetFile())); //uid aid pid did
		BufferedWriter bwmatrix = new BufferedWriter(new FileWriter(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "matrix"))); //uid mid r
		switch(method){
		case 4://Method 4: u,a,p,[u-p] -> d	
			
			//Write Fake mMap
			for(int p = 1;p <=Data.P;++p){mMap.put(new Long(p), p);}
			
			//Collect Matrix value: (#acc - #rej)/(#acc + #rej)
			int ucount = dataFold.data.uMap.size();
			double[][] acc = new double[ucount][Data.P];
			double[][] rej = new double[ucount][Data.P];
			double[][] hit = new double[ucount][Data.P];
			for(int u = 0;u < ucount;++u){Arrays.fill(acc[u], 0);Arrays.fill(rej[u], 0);Arrays.fill(hit[u], 0);}
			while((line=br.readLine())!=null){
				String[] frags = line.split(" ");
				uid = Integer.parseInt(frags[0]);
				pid = Integer.parseInt(frags[2]);
				mid = pid;//getM_id(method, aid, pid);
				did = Integer.parseInt(frags[3]);
				if(did==1){acc[uid-1][mid-1]++; hit[uid-1][mid-1]=1;}
				else if(did==-1){rej[uid-1][mid-1]++;hit[uid-1][mid-1]=1;}
			}
			//Write Matrix
			for(int u = 0;u < ucount;++u){
				for(int m = 0;m < Data.P;++m){
					if(hit[u][m]==1){
						double value = (acc[u][m] - rej[u][m]) / (acc[u][m] + rej[u][m]);
						bwmatrix.write((u+1)+" "+(m+1)+" "+value+"\n");
					}
				}
			}
			break;
			
		case 5://Method 5: u,a,p,[u-ap] -> d
			
			//Write Matrix and collect mMap
			//Matrix value: +1/0/-1
			while((line=br.readLine())!=null){
				String[] frags = line.split(" ");
				uid = Integer.parseInt(frags[0]);
				aid = Integer.parseInt(frags[1]);
				pid = Integer.parseInt(frags[2]);
				did = Integer.parseInt(frags[3]);
				m_id = getM_id(method, aid, pid);
				if(!mMap.containsKey(m_id)){mMap.put(m_id, mMap.size()+1);}
				mid = mMap.get(m_id);
				
				bwmatrix.write(uid+" "+mid+" "+did+"\n");
			}
			break;
			
		case 6://Method 6: u,a,p,[u-ap] -> d (Top apps only)
			 
			//Write Matrix and collect mMap
			//Matrix value: +1/0/-1
			
			while((line=br.readLine())!=null){
				String[] frags = line.split(" ");
				
				aid = Integer.parseInt(frags[1]);
				if(!dataFold.data.topASet.contains(aid)){continue;}//Only train on top 100 apps
				
				uid = Integer.parseInt(frags[0]);
				pid = Integer.parseInt(frags[2]);
				did = Integer.parseInt(frags[3]);
				m_id = getM_id(method, aid, pid);
				if(!mMap.containsKey(m_id)){mMap.put(m_id, mMap.size()+1);}
				mid = mMap.get(m_id);
				
				bwmatrix.write(uid+" "+mid+" "+did+"\n");
			}
			break;
			
		default:
			br.close();
			bwmatrix.close();
			throw new IOException("wrong method");	
		}
		//Write a fake record to make sure that the range of users and apps are matched.
		bwmatrix.write((dataFold.data.uMap.size()+1)+" "+(mMap.size()+1)+" "+0);
		br.close();
		bwmatrix.close();
		
		Tools.saveMap(mMap, getMFile(method, dataFold.v, dataFold.data.getTempDir(), "map"));
		
		
		
		//Generate Record for SVD regression (uid, mid)
		HashSet<UM> ums = new HashSet<UM>();
		genRecord(method, mMap, dataFold.getTrainSetFile(),ums);
		genRecord(method, mMap, dataFold.getTestSetFile(),ums);
		BufferedWriter bwrecord = new BufferedWriter(new FileWriter(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "record")));
		for(UM um:ums){
			bwrecord.write(um.uid+" "+um.mid+"\n");
		}
		bwrecord.close();
		
		//Remove previous results
		new File(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "uf")).delete();
		new File(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "mf")).delete();
		new File(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "prediction")).delete();
		new File(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "log")).delete();
		
		scripts.add(sg.genRIrlba(N, 
				getMFile(method, dataFold.v, dataFold.data.getTempDir(), "matrix"), getMFile(method, dataFold.v, dataFold.data.getTempDir(), "uf"),
				getMFile(method, dataFold.v, dataFold.data.getTempDir(), "mf"), getMFile(method, dataFold.v, dataFold.data.getTempDir(), "record"), 
				getMFile(method, dataFold.v, dataFold.data.getTempDir(), "prediction"), getMFile(method, dataFold.v, dataFold.data.getTempDir(), "log")));
	}
	
	void genRecord(int method, HashMap<Long,Integer> mMap, String fin, HashSet<UM> ums) throws IOException{
		BufferedReader br = new BufferedReader(new FileReader(fin));
		int uid,aid,pid,mid;long m_id,e_id;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			aid = Integer.parseInt(frags[1]);
			pid = Integer.parseInt(frags[2]);
			m_id = getM_id(method, aid, pid);
			
			//Here we only calculate the uid-mid pairs appeared in SVD training.
			if(mMap.containsKey(m_id)){
				mid = mMap.get(m_id);
				ums.add(new UM(uid, mid));
			}
		}
		br.close();
	}
	
	
	
	
	
	
	
	
	
	
	
	
	
	public void run(String cmd, String tmpFolder,String label) throws IOException, InterruptedException{
		System.out.println("\texec script:\n\t\t"+cmd);
		
		Process p = Runtime.getRuntime().exec(cmd);
		p.waitFor();
		System.out.println("\tscript running");
		BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
		String line;
		while ((line=reader.readLine())!=null){System.out.println("\t\t"+line);}
		
		
		ScriptGen.SignalObserver so = new ScriptGen.SignalObserver(
				tmpFolder, label, Data.CORES, false);
		
		so.waitWhenFinish();
		System.out.println("\tsignalobserver finished");
	}
	
	
	public void collectGen(ScriptGen sg) throws IOException{
		List<String> scripts = new ArrayList<String>();
		
		for(DataFold dataFold:dataFolds){
			collectAndGenM(4, dataFold, scripts, sg);
			//collectAndGenM(5, dataFold, scripts, sg);
			collectAndGenM(6, dataFold, scripts, sg);
		}
		
		sg.splitScript(scripts, Data.CORES, "flow3train", "flow3train.sh",
				dataFolds.get(0).data.getScriptDir(), dataFolds.get(0).data.getTempDir());
	}
	void collectAndGenM(int method, DataFold dataFold, List<String> scripts, ScriptGen sg) throws IOException{
		
		//Collect SVD regression results
		HashMap<Long,Integer> mMap = Tools.loadMap(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "map"));
		List<Long> umids = new ArrayList<Long>(); // uid * mMap.size() + mid
		List<Double> rs = new ArrayList<Double>();
		
		BufferedReader br = new BufferedReader(new FileReader(getMFile(method, dataFold.v, dataFold.data.getTempDir(),  "record")));
		int uid, mid;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			mid = Integer.parseInt(frags[1]);
			umids.add(getE_id(uid, mid, mMap.size()));
		}
		br.close();
		br = new BufferedReader(new FileReader(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "prediction")));
		while((line=br.readLine())!=null){
			rs.add(Double.parseDouble(line));
		}
		br.close();
		if(umids.size() != rs.size()){throw new IOException("umids!=rs: "+umids.size()+" "+rs.size());}
		HashMap<Long, Double> svd = new HashMap<Long,Double>(); // uid * mMap.size() + mid -1 -> regression
		for(int i = 0;i < umids.size();++i){
			svd.put(umids.get(i), rs.get(i));
		}
		System.out.println("colectAndGen mMap.size="+mMap.size()+" svd.size="+svd.size());
		//Tools.saveMap(svd, getMFile(method, dataFold.v, dataFold.data.getTempDir(), "svd"));
		
		//If no match, use Method 4 matrix input as default value
		int uMapSize = dataFold.data.uMap.size();
		double[][] upavg = new double[uMapSize][Data.P];
		for(int i = 0;i < uMapSize;++i){Arrays.fill(upavg[i], 0);}
		
		br = new BufferedReader(new FileReader(getMFile(4, dataFold.v, dataFold.data.getTempDir(), "matrix")));
		int pid;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			if(uid > uMapSize || pid > Data.P){continue;}
			upavg[uid-1][pid-1] = Double.parseDouble(frags[2]);
		}
		br.close();
		
		/*
		//Add real sparse matrix values to the regression results
		br = new BufferedReader(new FileReader(getMFile(method, dataFold.v, dataFold.data.getTempDir(), "matrix")));
		long e_id;double d;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			mid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			e_id = getE_id(uid, mid, mMap.size());
			svd.put(e_id, d);
		}
		br.close();
		*/
		
		
		
		String f_initTrain = Flow2.getMFile(dataFold.v, dataFold.data.getTempDir(), 3, "train");
		String f_initTest = Flow2.getMFile(dataFold.v, dataFold.data.getTempDir(), 3, "test");
		
		String f_newTrain = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "lltrain");
		String f_newTest = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "lltest");
		String f_newModel = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "llmodel");
		String f_newOut = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "llout");
		String f_newLog = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "lllog");
		
		new File(f_newModel).delete();
		new File(f_newOut).delete();
		new File(f_newLog).delete();
		
		//Add regression as additional feature to the svm inputs
		double hitTrain = addRegression(f_initTrain, f_newTrain, method, mMap, svd, dataFold.data, upavg);
		double hitTest = addRegression(f_initTest, f_newTest, method, mMap, svd, dataFold.data, upavg);
		System.out.println("collectAndGen m"+method+" v"+dataFold.v+" hitTrain="+hitTrain+" hitTest="+hitTest);
		
		scripts.add(sg.genLibLinear(f_newTrain, f_newTest, f_newModel, f_newOut, f_newLog));
	}
	
	double addRegression(String f_in, String f_out, int method, HashMap<Long, Integer> mMap, HashMap<Long, Double> svd, Data data, double[][] upavg) throws IOException{
		int uid,aid,pid,mid; long m_id; double d,r;String line;
		BufferedReader br = new BufferedReader(new FileReader(f_in));
		BufferedWriter bw = new BufferedWriter(new FileWriter(f_out));
		
		double totalHit = 0;
		double totalCount = 0;
		while((line=br.readLine())!=null){
			
			
			String newLine = Flow2.translateFeatures(data, line, 3);
			//System.out.println(line+"\t"+newLine);
			//if(line!=null){return 0;}
			String[] frags = newLine.split(" "); //did, pid, aid, uid
			
			uid = Integer.parseInt(frags[3]);
			aid = Integer.parseInt(frags[2]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[0]);
			
			m_id = getM_id(method, aid, pid);
			
			
			totalCount += 1;
			int featureIndex = Data.P + data.aMap.size() + data.uMap.size() + 1;
			if(mMap.containsKey(m_id)){
				
				mid = mMap.get(m_id);
				long e_id = getE_id(uid, mid, mMap.size());	
				
				if(svd.containsKey(e_id)){//If we have the svd regression, add as an additional feature
					totalHit += 1;
					r = svd.get(e_id);
					bw.write(line+" "+featureIndex+":"+r+"\n");
				}else{
					bw.write(line+" "+featureIndex+":"+upavg[uid-1][pid-1]+"\n");
				}
			}else{
				bw.write(line+" "+featureIndex+":"+upavg[uid-1][pid-1]+"\n");
			}
		}
		//System.out.println("addRegression hitRate="+(totalHit/totalCount));
		br.close();
		bw.close();
		return totalHit / totalCount;
	}
	
	
	public void collectResult() throws IOException{
		String resultDir = dataFolds.get(0).data.getResultDir();
		collectResultM(4, getMResultFile(4, resultDir));
		//collectResultM(5, getMResultFile(5, resultDir));
		collectResultM(6, getMResultFile(6, resultDir));
	}
	void collectResultM(int method, String resultFile) throws IOException{
		double totalAcc = 0;
		BufferedWriter bw = new BufferedWriter(new FileWriter(resultFile));
		for(DataFold dataFold:dataFolds){
			
			String logFile = getMFile(method, dataFold.v, dataFold.data.getTempDir(), "lllog");
			BufferedReader br = new BufferedReader(new FileReader(logFile));
			String line = br.readLine();
			double acc = Double.parseDouble(line.substring(line.indexOf('=')+1, line.indexOf('%')));
			bw.write(acc+"\t");
			totalAcc += acc;
			br.close();
		}
		bw.write("\n"+totalAcc/dataFolds.size()+"\n");
		bw.close();
	}
	
}
